import re
import os
import time
import base64
import urlparse
import md5
import urllib
import urllib2
import httplib
#from uriparse import urljoin
from datetime import datetime



def Save(html):
        f = open("out.html", 'w')
        f.write(html)
        f.close()
def upanh(link):
        
        #try:
                key = md5.md5(link + 'socbaynews@uploadanh.com' + 'UPANH_API__').hexdigest()
                return link
                parameter = base64.b64encode(link) + '|6054|' + key       
                parameter = "pnews=" + parameter
                print parameter
                reg = urllib2.Request("http://upanh.com/cache_post2.php?getlink", parameter)
                print 1
                html = urllib2.urlopen(reg).read()
                print html
                pos = html.find('<vienok>')
                pos1 = html.find('</vienok>', pos + 1)
                if pos == -1 or pos1 == -1:                        
                        return link
                return html[pos + len('<vienok>'):pos1]
        #except:
                return link
def findtags(pos, html):
	pos1 = pos
	while pos > 0 :
		if html[pos] == '<':
			break
		pos = pos - 1
	start = pos + 1
	while pos < pos1 :
		if html[pos] == ' ' :
			break
		pos = pos + 1
	return html[start:pos]        
def replace_tinlienquan(html):
	import re
	reg = re.compile('(([<][aA][^>]+[>])(.*?)([<][/][Aa][>]))')
	x = reg.findall(html)
	for i in x:
		if i[2].find('<img') == -1 and i[2].find('<IMG') == -1:
			html = html.replace(i[0], '')
		else:
			html = html.replace(i[0], i[2])
	return html
def UpDateImages(content, url):
	import re	
	reg = re.compile('src\\s*=\\s*[\'"]([^\'"]+)[\'"]')
	Img = reg.findall(content)
	link = ''
	for h in Img:		
		link = AddImg(h, url)
		content = content.replace(h, link)
	return content
def remove_spen(html, pos):
	while pos > 0:
		if html[pos] == '>':
			return 1
		if html[pos] == '<':
			return -1
		pos = pos - 1
	return -1
def convertTime24hHMDMY(timeDate):
	reg = re.compile('([0-9]+)[:]([0-9]+).*?([0-9]+)[/]([0-9]+)[/]([0-9]+).*?')
	time_ = "";
	#print timeDate
	if reg.search(timeDate):
		time_ = numberFomat(reg.search(timeDate).group(1)) + ':' + numberFomat(reg.search(timeDate).group(2)) + ' ' + numberFomat(reg.search(timeDate).group(3)) + '-' + numberFomat(reg.search(timeDate).group(4)) + '-' + checkYear(reg.search(timeDate).group(5))
		
	else:
		print 'loi ham thoi gian'
		return ' '
	return time_
def replace_script(html):
	while True:
		relate = s_string(html, 'type="text/javascript"', 'script')
		if len(relate) == 0:
			break
		html = html.replace(relate, '')
	while True:
		relate = s_string(html, 'language="JavaScript"', 'script')
		if len(relate) == 0:
			break
		html = html.replace(relate, '')
	while True:
		relate = s_string(html, 'language="javascript"', 'script')
		if len(relate) == 0:
			break
		html = html.replace(relate, '')
	
	return html
def convertTimePM_d_m_y_H_M(timeDate):
	from datetime import datetime
	import re
	s = __import__("pythonlibSB")
	reload(s)
	time_ = ' '
	#print timeDate
	reg1 = re.compile('([0-9]+)[/]([0-9]+)[/]([0-9]+).*?([0-9]+)[:]([0-9]+).*?[PM]')
	chieu = 0;
	if reg1.search(timeDate):
		chieu = 12;        
	reg = re.compile('([0-9]+)[/]([0-9]+)[/]([0-9]+).*?([0-9]+)[:]([0-9]+)')
	
	if reg.search(timeDate):
		if chieu != 0:
			timeDate = s.checkNum(reg.search(timeDate).group(4)) + ':' + reg.search(timeDate).group(5) + ' PM ' + reg.search(timeDate).group(1) + '-' + reg.search(timeDate).group(2) + '-' + reg.search(timeDate).group(3)
		else:
			timeDate = s.checkNum(reg.search(timeDate).group(4)) + ':' + reg.search(timeDate).group(5) + ' AM ' + reg.search(timeDate).group(1) + '-' + reg.search(timeDate).group(2) + '-' + reg.search(timeDate).group(3)
	else:
		print 'loi khong tach duoc thoi gian'
		return 1
	try:
		dt = datetime.strptime(timeDate, "%I:%M %p %d-%m-%Y")
		time_ = dt.strftime("%H:%M %d-%m-%Y")
	except:
		print 'loi thoi gian;'
		return 1
	return time_

def remove_spencel(html):
	pos = 0
	kt = 0
	while pos < len(html):
		pos = html.find('>', pos + 1)
		if pos == -1:
			break
		kt = remove_spen(html, pos - 1)		
		if kt == 1:
			html = html[0:pos] + html[pos + 1:]
			pos = pos - 6
			
	return html

def setTimeCache(timeCache, timeItem):
	from  datetime import datetime
	try:
                ret = datetime.strptime(timeCache, "%H:%M %d-%m-%Y")
                ret1 = datetime.strptime(timeItem, "%H:%M %d-%m-%Y")
                r = ret1 - ret
                kt = (r.days * 24 * 60) + (r.seconds / 60)
                if kt > 0:
                        return 1 #tuc la phai gan lai thoi gian                
                return 0
        except:
                return -1
def remove_scr(content):
	while True:
		relate = s_string(content, 'script', 'script')
		if len(relate) == 0:
			break
		content = content.replace(relate, '')
	return content
def getNow_s(timeDate):
	
	now = datetime.now()
	reg = re.compile('([0-9]+)[-]([0-9]+)[-]([0-9]+)')
	if reg.search(timeDate):
		return now.strftime("%H:%M") + ' ' + checkNum(reg.search(timeDate).group(1)) + '-' + checkNum(reg.search(timeDate).group(2)) + '-' + reg.search(timeDate).group(3)
        return ''	    
def getTimeSpan():
    from  datetime import datetime
    ret = datetime.now() - datetime.strptime("01-01-1990", "%d-%m-%Y")
    return (ret.days * 24 * 60) + (ret.seconds / 60)

def remove_encode(html):
	reg = re.compile('(&[^;]+;)')
	x = reg.findall(html)
	for i in x:
		if len(i) < 12:
			html = html.replace(i, '')
	return html
	
def checkNum(strnum):
	if len(strnum) == 1:
		return '0' + strnum
	else:
		return strnum
def checkYear(strYear):
	if len(strYear) == 2:
		return '20' + strYear
	else:
		return strYear
def numberFomat(num):
	if len(num) == 1:
		return '0' + num
	else:
		return num
def ConvertContent_(content, url):		
	content = re.sub('<si>[^<]+</si>', '', content)
	content = re.sub('<sp>', '', content)
	content = re.sub('</sp>', '', content)	
	return  base64.b64encode(content)
def Upanh_(url, titleImg, title, HostName):
        widthHeigh = " | ";
	ilink = url
	return upanh(url)
	data = {'a':removeSpecialChar(titleImg) + "|" + removeSpecialChar(title) + "|" + base64.b64encode(url) + "|" + HostName + "|" + widthHeigh + "|" + SinhKey(url) + "|6054" }	
	body = urllib.urlencode(data)
	body = body[2:]	
	parameter = "pnews=" + body		
	conn = httplib.HTTPConnection("ver2.uploadanh.com", 80, 30)
	headers = {"Content-Type": "application/x-www-form-urlencoded", "Keep-Alive":"115", "Connection":"keep-alive", "User-Agent": "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.6) Gecko/20050512 Firefox", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Host": "ver2.uploadanh.com"}        
	conn.request("POST", "/cache_post.php?getlink", parameter, headers)
	response = conn.getresponse()
	html = response.read()
	print html
	if html.find('<vienok>') != -1:
		pos = html.find('<vienok>')
		pos1 = html.find('</vienok>')
		return html[pos + len('<vienok>'):pos1]
	else:
		return ilink
	return ilink
def UpLoadAnh(url, titleImg, title, HostName):
        urllink = ''
	try:
                urllink = Upanh_(url, titleImg, title, HostName)
                return urllink
        except:
                urllink = Upanh_(url, "", "", HostName)
                return urllink
        return url
                
def SinhKey(path):
	md5_ = "http://c.uploadanh.com/cache_post.php?url=" + path + "&user=" + "socbaynews@uploadanh.com";
	import md5
	key = md5.md5(md5_).hexdigest()
	return key
def removeSpecialChar(inp):
        if len(inp) == 0:
                return inp
	inp = inp.replace("|", ';')
	return inp
def GetImages_(newS, key, tag):       	
	content = newS.content
	temp = content
	import re
	title = ''
	hostName = ''
	try:
		title = newS.title
	except:
		title = newS.title

	try:
		hostName = newS.url
	except:
		hostName = newS.url	
	
	reg = re.compile('src\\s*=\\s*[\'"](.*?)[\'"]+')
	h = s_string_2(temp, key, tag)	
	ktimg = 0
	while len(h) > 0:		
		temp = temp.replace(h, '')
		if reg.search(h):
			images = reg.search(h).group(1)			
			titleImg = remove_tag(h)
			_images = AddImg(images, hostName)
			print '_titleImg: ' + titleImg
			link = _images
			try:
                                link = UpLoadAnh(_images, titleImg, title, hostName)

                        except:
                                try:
                                        link = UpLoadAnh(_images, ' ', ' ', newS.hostName)
                                except:
                                        print 'loi link'
                                print 'loi link khi cache'
                                link = _images
			print 'link' + link
			if ktimg == 0 :
				newS.thumbUrl = link
				ktimg = ktimg + 1
			content = content.replace(images, link)
			file_ = newS.fileAttach.add()
			file_.url = _images
			file_.urlCache = link
			try:
				file_.title = newS.title
			except:
				print 'loi cho nay roi '
			file_.fileType = 5
		h = s_string_2(temp, key, tag)
		if len(h) == 0:
                    break;
	newS.content = content
	return newS
def GetImages(newS):
	import re	
	reg = re.compile('src\\s*=\\s*[\'"](.*?)[\'"]')
	content = newS.content       
	temp = content	
	title = ''
	hostName = ''
	title = newS.title
	url = newS.url	
	ktimg = 0
	Img = reg.findall(content)
	print len(Img)
	titleImg = ' '
	
	for h in Img:
		_images = AddImg(h, url)
		print '_images' + _images
		link = _images
		
		try:
                        link = UpLoadAnh(_images, titleImg, title, newS.hostName)
                except:
                        try:
                                link = UpLoadAnh(_images, ' ', ' ', newS.hostName)
                        except:
                                print 'loi link'  
                        print 'loi link'                                
		print 'link' + link
		try:
                        if ktimg == 0 :
                                newS.thumbUrl = link
                                ktimg = ktimg + 1
                except:
                        print 'loi roai'
		content = content.replace(h, link)
		try:
                        file_ = newS.fileAttach.add()
                        file_.url = _images
                        file_.urlCache = link
                        try:
                                file_.title = newS.title
                        except:
                                print 'loi cho nay roi '
                        file_.fileType = 5
                except:
                        print 'loi roi'        
	newS.content = content
	return newS
def getNow_1(timeDate):	
	now = datetime.now()
	reg = re.compile('[ ]([0-9]+)[ ].*?[ ]([0-9]+)[ ].*?[ ]([0-9]+)')
	if reg.search(timeDate):
		return now.strftime("%H:%M") + ' ' + checkNum(reg.search(timeDate).group(1)) + '-' + checkNum(reg.search(timeDate).group(2)) + '-' + reg.search(timeDate).group(3)
        return ''	
def getNow_(timeDate):	
	now = datetime.now()
	reg = re.compile('([0-9]+)[/]([0-9]+)[/]([0-9]+)')
	if reg.search(timeDate):
		return now.strftime("%H:%M") + ' ' + checkNum(reg.search(timeDate).group(1)) + '-' + checkNum(reg.search(timeDate).group(2)) + '-' + reg.search(timeDate).group(3)
        return ''            
def convertTimenhacviet(timeDate):
	reg = re.compile('([0-9]+)[/]([0-9]+)[/]([0-9]+).*?([0-9]+)[:]([0-9]+).*?')
	time_ = "";
	#print timeDate
	if reg.search(timeDate):
		time_ = checkNum(reg.search(timeDate).group(4)) + ':' + checkNum(reg.search(timeDate).group(5)) + ' ' + checkNum(reg.search(timeDate).group(1)) + '-' + checkNum(reg.search(timeDate).group(2)) + '-' + checkNum(reg.search(timeDate).group(3))
		
	else:
		print 'loi ham thoi gian'
		return ' '
	return time_
def convertTime_d_m_y_h_m(timeDate):
	reg = re.compile('([0-9]+)[/]([0-9]+)[/]([0-9]+).*?([0-9]+)[:]([0-9]+).*?')
	time_ = "";
	#print timeDate
	if reg.search(timeDate):
		t3 = reg.search(timeDate).group(5)
		if len(t3) == 1:
			t3 = '0' + t3                        
		time_ = numberFomat(reg.search(timeDate).group(4)) + ':' + numberFomat(t3) + ' ' + numberFomat(reg.search(timeDate).group(1)) + '-' + numberFomat(reg.search(timeDate).group(2)) + '-' + reg.search(timeDate).group(3)
		
	else:
		print 'loi ham thoi gian'
		return ' '        
	return time_
def convertTime24h(timeDate):
	reg = re.compile('([0-9]+)[/]([0-9]+)[/]([0-9]+).*?([0-9]+)[:]([0-9]+).*?')
	time_ = "";
	#print timeDate
	if reg.search(timeDate):
		time_ = numberFomat(reg.search(timeDate).group(4)) + ':' + numberFomat(reg.search(timeDate).group(5)) + ' ' + numberFomat(reg.search(timeDate).group(1)) + '-' + numberFomat(reg.search(timeDate).group(2)) + '-' + checkYear(reg.search(timeDate).group(3))
		
	else:
		print 'loi ham thoi gian'
		return ' '
	return time_
def convertTimeDOT(timeDate):
	reg = re.compile('([0-9]+)[.]([0-9]+)[.]([0-9]+).*?([0-9]+)[:]([0-9]+).*?')
	time_ = "";
	#print timeDate
	if reg.search(timeDate):
		time_ = numberFomat(reg.search(timeDate).group(4)) + ':' + numberFomat(reg.search(timeDate).group(5)) + ' ' + numberFomat(reg.search(timeDate).group(1)) + '-' + numberFomat(reg.search(timeDate).group(2)) + '-' + reg.search(timeDate).group(3)
		
	else:
		print 'loi ham thoi gian'
		return ' '
	return time_
#db=_mysql.connect(host="10.0.14.16",port=3306,user="root",passwd="bimat",db="news_db")
def convertPMTime(timeDate):
	reg1 = re.compile('([0-9]+)[:]([0-9]+).*?[P].*?([0-9]+)[/]([0-9]+)[/]([0-9]+)')
	chieu = 0;
	if reg1.search(timeDate):
		chieu = 12;        
	reg = re.compile('([0-9]+)[:]([0-9]+).*?([0-9]+)[/]([0-9]+)[/]([0-9]+)')
	
	if reg.search(timeDate):
		k = reg.search(timeDate).group(1)
		k = int(k)
		k = k + chieu
		k = str(k)
		if len(k) == 1:
			k = '0' + k                
		timeDate = k + ':' + numberFomat(reg.search(timeDate).group(2)) + ' ' + numberFomat(reg.search(timeDate).group(3)) + '-' + numberFomat(reg.search(timeDate).group(4)) + '-' + checkYear(reg.search(timeDate).group(5))
	else:
		print 'loi khong tach duoc thoi gian'
		return ''
	return timeDate

def convertTime_y_m_d_h_m(timeDate):
	reg = re.compile('([0-9]+)[/]([0-9]+)[/]([0-9]+).*?([0-9]+)[:]([0-9]+).*?')
	time_ = "";
	#print timeDate
	if reg.search(timeDate):
		time_ = reg.search(timeDate).group(4) + ':' + reg.search(timeDate).group(5) + ' ' + reg.search(timeDate).group(3) + '-' + reg.search(timeDate).group(2) + '-' + reg.search(timeDate).group(1)
		
	else:
		print 'loi ham thoi gian'
		return ' '
	return time_

def getNow():	
	now = datetime.now()
	return now.strftime("%H:%M %d-%m-%Y")
def getNow1():	
	now = datetime.now()
	return now.strftime("%d/%m/%Y")  

def checkRemove(html):
	for i in range(len(html)):
		if html[i] == '<' and html[i + 1] != ' ':
			kt = i
			for j in range(i, len(html) - 1):
				kt = j
				if html[j] == ' ' or html[j] == '>':
					break
			if len(html[i:kt]) != 0 and html[i + 1] == '/':
				#tim duoc mot the moi,gan ten the vua tim duoc vao tag
				tag = html[i:kt + 1]

				reg = re.compile('csbay.*?')
				if reg.search(tag):
					continue
				else:
					html = checkTag(html, i, "fs-bay")
	html = re.sub('<fs-bay/[^>]+>', '', html)
	html = re.sub('<fsbay[^>]+>', '', html)
	html = re.sub('csbay', '', html)
	return html
def s_index_string(html, key, pos):
	a = html.find(key, pos)
	a = first_index(html, a)		 			# tim den dau '<
	b = index_of_tag(html, key, a - 1) 	# tim vi tri ket thuc cua the tag
	return b
def checkTag(html, pos, push):
	pos = pos + 1
	temp = html
	html = temp[0:pos] + push + temp[pos:len(html)]
	return html
def checkTrue(tag):
	R = ['img', 'IMG', 'embed', 'EMBED', 'em', 'EM', 'br', 'BR']
	for i in R:
		if tag == i:
			return -1
	return 1    
def Remove_html(html):
	n = len(html)
	for i in range(0, n):
		if i > n - 1:
			break;
		if html[i] == '<':
			kt = i
			for j in range(i, len(html)):
				kt = j
				if html[j] == ' ' or html[j] == '>':
					break
			if len(html[i:kt]) != 0 and html[i + 1] != '/':
				#tim duoc mot the moi,gan ten the vua tim duoc vao tag
				tag = html[i + 1:kt]
				if checkTrue(tag) == 1:
					#tim den vi tri ket thuc cua the nay
					posEnd = s_index_string(html, tag, i)
					if posEnd != -1:
						html = checkTag(html, posEnd, "csbay")
						n = len(html) - 1

					else:
						temp = html
						html = temp[0:i] + temp[kt + 1:len(html)]
						n = len(html) - 1



	html = checkRemove(html)
	return html

def isInt(a):
	import re	
	reg = re.compile('([0-9]+)')	
	if reg.search(a):
		return int(reg.search(a).group(1))
	else:
		return -1
def index_of_tag(xau, tag, pos):
	i = pos
	k = 0
	while(i < len(xau) and k >= 0):
		i = xau.find(tag, i + 1)
		if(i > 0):
			if(xau[i - 1] == '<'):
				k = k + 1
			else:
				if(xau[i - 1] == '/' and xau[i - 2] == '<'):
					k = k - 1
					if(k == 0):
						return i - 1
		else:
			break
	if(k == 0):
		return i
	return -1
# chay ve dau cua the
def first_index(xau, pos):
	i = pos
	while(i > 0):
		i = i - 1
		if(xau[i] == '<'):
			break
	if(i > -1):
		return i
	return -1
	
#chay ve cuoi cua the
def last_index(xau, pos):
	i = pos
	while(i < len(xau) - 1):
		i = i + 1
		if(xau[i] == '>'):
			break
	if(i < len(xau) + 1):
		return i
	return -1
def remove_tag(html):        
    html = html.replace('\"', '"')
    html = html.replace("\'", "'")
    html = re.sub('<.*?>', ' ', html)
    html = html.strip()
    return html
def remove_script(html):
	html = html.replace('\r\n', ' ')
	while True:                
		pos = html.find('<script>')
		pos1 = html.find('</script>')
		if pos == -1 or pos1 == -1:
			break
		html = html[0:pos] + html[pos1 + len('</script>'):len(html)]        
	#khi da het bat dau va ket thuc
	html = re.sub('<script.*?>', ' ', html)
	html = re.sub('<[Ss][Cc][Rr][Ii][Pp][Tt].*?>.*?</[Ss][Cc][Rr][Ii][Pp][Tt]>', '', html)
	#<style
	html = re.sub('<[Ss][Tt][Yy][Ll][Ee].*?>.*?</[Ss][Tt][Yy][Ll][Ee]>', '', html)
	html = re.sub('<[Ss][Tt][Yy][Ll][Ee]>.*?</[Ss][Tt][Yy][Ll][Ee]>', '', html)
	#loai bo lien ket
	#html=re.sub('<a.*?>.*?</a>','',html)
	while True:
		pos = html.find('script')
		if pos == -1:
			break
		pos1 = html.find('>', pos)
		if pos1 == -1:
                        break;
		html = html[0:pos - 1] + html[pos1:len(html)]
	while True:
		pos = html.find('<st1')
		if pos == -1:
			break
		pos1 = html.find('>', pos)
		if pos1 == -1:
                        break;  
		html = html[0:pos] + html[pos1:len(html)]
	return html
	return ' '

 
# ham lay toan bo noi dung cua mot the
def s_string(html, key, tag): 		# html la xau truyen vaotag la ten tag vi du div key la lop hoac id vi du class=Lead
	a = html.find(key)							# tim kiem ten cua lop trong html
	tag = findtags(a, html)
	a = first_index(html, a)		 			# tim den dau '<' dau tien ke tu vi tri tim duoc
	b = index_of_tag(html, tag, a - 1) 	# tim vi tri ket thuc cua the tag
	b = last_index(html, b)
	return html[a:b + 1]
def first_index_tag(html, tag, pos):
	i = pos
	while i > 6:
		temp = html[i:pos]
		if temp.find(tag) != -1:
			return i - 1
		i = i - 1
	return 0
def s_string_2(html, key, tag):#ham nay duoc viet ra de lay trong truong hop ham 1 khong thuc hien duoc 
	a = html.find(key)
	a = first_index_tag(html, tag, a)
	b = index_of_tag(html, tag, a - 1)
	b = last_index(html, b)
	return html[a:b + 1]

def s_test(html, key, tag): 		# html la xau truyen vaotag la ten tag vi du div key la lop hoac id vi du class=Lead
	a = html.find(key)							# tim kiem ten cua lop trong html
	a = first_index(html, a)		 			# tim den dau '<' dau tien ke tu vi tri tim duoc
	b = index_of_tag(html, tag, a - 1) 	# tim vi tri ket thuc cua the tag
	b = last_index(html, b)
	return b

def AddImg(img, link):
    return link
'''        from uriparse import urljoin
        urlabs = urljoin(link, img)
        return urlabs
'''
def imgTemp(html, HostName):
	
	reg = re.compile('src\\s*=[\'"](.*?)[\'"]+') 
	s = reg.findall (html)	
	for i in s:
		html = html.replace(i, AddImg(i, HostName))
		
	return html
def Insert(html, pos, tag):
	return html[0:pos] + tag + html[pos:len(html)]
def Delete(html, pos, poslen):
	return html[0:pos] + html[pos + poslen:len(html)]
def chuanHoaStr(html):
	html = Insert(html, 0, '<pasp>')
	ktr = 0
	i = 6
	while(i < len(html) - 8):
                ktr = ktr + 1
                if ktr == 100000:
                        break
		kt = 0
		if html[i] == '<' and html[i + 1] != '/':#neu la the bat dau thi chen 1 the ket thuc
			kt = 1
			html = Insert(html, i, '</pasp>')
			i = i + 11
		if html[i] == '<' and html[i + 1] == '/' and html[i + 2] == 'p' and html[i + 3] == 'a' and html[i + 4] == 's' and html[i + 5] == 'p':#neu gap the ket thuc thi xoa
			kt = 1
			#html =Delete(html,i,len('</pasp>'))
			html = Insert(html, i + 7, '<pasp>')
			i = i + 12

		if html[i] == '<' and html[i + 1] == '/' and html[i + 2] == 'p' and html[i + 3] == 'a' and html[i + 4] == 's' and html[i + 5] == 'i':#neu gap thi ket thuc anh thi them vao the bat dau
			kt = 1
			html = Insert(html, i + 7, '<pasp>')
			i = i + 12
		if kt == 0:
			i = i + 1
	html = re.sub('<pasp>\\s*</pasp>', '', html)
	return html
def Remove_allContent(html):
	n = len(html) - 1
	for i in range(n):
		if i > len(html) - 4:
			break;
		if (html[i] == '<'):
			pos = last_index(html, i)
			t = html[i:pos + 1]
			pos = t.find('si>')
			if pos != -1:
				continue
			pos = t.find('sp')
			if pos != -1:
				continue
			html = html.replace(t, '')
			n = n - len(t)
	return html
def contentMobile(html, HostName):
	html = html.replace('\r\n', '')

	reg = re.compile('<[iI][mM][gG][^>]+src[^>]+>')
	s = reg.findall (html)
	reg = re.compile('src\\s*=\\s*[\'"](.*?)[\'"]')
	for i in s:
		if reg.search(i):
			html = html.replace(i, '<si>' + AddImg(reg.search(i).group(1), HostName) + "</pasi>")

	Replace = ['div', 'hr', 'p', 'P', 'DIV', 'HR', 'table', 'TABLE', 'tr', 'td', 'TR', 'TD']
	#Replace=['div','span','b','B','p','P','DIV','SPAN']
	for i in Replace:
		html = re.sub('<' + i + ' .*?>', '<pasp>', html)
		html = re.sub('<' + i + '>', '<pasp>', html)
		html = re.sub("</" + i + ">", "</pasp>", html)
		
	html = re.sub('<\?xml:.*?>', '', html)               
	html = re.sub('<[Ss][Pp][Aa][n]>', '', html)
	html = re.sub('<[Ss][Pp][Aa][n].*?>', '', html)
	html = re.sub('</[Ss][Pp][Aa][n]>', '', html)
	html = re.sub("<br.*?>", "</pasp>", html)
	html = re.sub("</[Tt][Aa][Bb][Ll][Ee]>", "", html)
	html = re.sub("<br>", "</pasp>", html)
	html = re.sub('<[aA].*?>', '', html)
	html = re.sub('<[Hh].*?>', '', html)
	html = re.sub("</[Hh]>", '', html)
	html = re.sub('</[Hh].*?>', '', html)
	html = re.sub('</[Hh].*?>', '', html)
	html = re.sub("<[Ee].*?>", '', html)	
	html = html.replace('<si>', '<pasi>')
	html = Remove_allContent(html)
	html = chuanHoaStr(html)
	html = html.replace('pas', 's')
	html = re.sub('[\\s][\\s]+', ' ', html)
	html = html.replace('&lt;', '<')
	html = html.replace('&gt;', '>')
	return html


def extractFind(html):
	html = html.replace('\r\n', '')
	content = remove_tag(html)
	pos = content.find('.')
	if pos == -1:
		return ' '
	if pos > 100:
		return content[0:pos]
	extract = content[0:pos]
	pos1 = pos
	while True:            
		if html.find(extract) == -1:                
			return extract
		extract = html[0:pos1]
		pos1 = content.find('.', pos1 + 3)
		if pos1 == -1:                
			return extract
	    
def timeToIntAll(string):	
	datetimeRoot = datetime.strptime('00:00 01-01-1970', '%H:%M %d-%m-%Y')
	datetimeNow = datetime.strptime(string.strip(), '%d-%m-%Y %H:%M')
	a = datetimeNow - datetimeRoot
	return a.days * 3600 * 24      

def ReplaceHtmlSpecial(html):	
	return html
#yeu cau dau vao
#time_: da duoc chuan hoa ,content de nguyen tag, extract de nguyen ,imglink= ' ',strconfig la strconfig, html la file noi
def getHtmlId(h, IdItem, hostId):
	import urllib 
	htmlId = ''
	try:
		t = 'http://10.0.12.95:12266/ServiceID.aspx?fn=simdetect&hash=' + str(h) + '&id=' + str(IdItem) + '&hid=' + str(hostId)
		print t
		htmlId = urllib.urlopen(t, proxies={}).read()                
	except:
		print 'loi khong lay duoc level'
		return ' '
	return htmlId
def getID():        
	html = '0'       
	try:
		html = urllib.urlopen('http://10.0.12.95:12266/ServiceID.aspx?fn=getid', proxies={}).read()
	except:
		print'loi lay ID'       
	return html
def checkExtract(html, extract):
	if html.find(extract) == -1:
		return -1;
	return  1
def getExtract(html):
	html = html.replace('\r\n', '')
	content = remove_tag(html)
	for i in range(0, len(content)):
		if checkExtract(html, content[0:i]) == -1:
			return content[0:i - 1]
def getExtract_2(html):
	html = html.replace('\r\n', '')
	content = remove_tag(html)
	extract = ''	
	extract_str = ''
	while len(content) > 200:
		for i in range(0, len(content)):
			if checkExtract(html, content[0:i]) == -1:
				extract = content[0:i - 1]				
				break
		extract_str = extract_str + extract
		if len(extract_str) > 20:
			return extract_str
		content = content[i:len(content)]
	return extract_str

def findF(content, pos):
	pos1 = pos
	pos2 = -1
	pos3 = -1
	while pos1 > 0:
		pos1 = pos1 - 1
		if pos2 != -1:
			return 1
		if pos3 != -1:
			return -1
		if content[pos1] == '>':
			pos2 = pos1
		if content[pos1] == '<':
			pos3 = pos1
	return -1
def replaceF(content):	
	n = len(content)
	pos = 0
	while pos < n:
		pos = content.find('>', pos + 1)
		if pos == -1:
			break
		if findF(content, pos) == 1:
			content = content[:pos] + content[pos + 1:]
			pos = pos - 2
	return content                        
		
def convertPMTime123(timeDate):
	reg1 = re.compile('([0-9]+)[/]([0-9]+)[/]([0-9]+).*?([0-9]+)[:]([0-9]+).*?[P].*?')
	chieu = 0;
	if reg1.search(timeDate):
		chieu = 12;
	reg = re.compile('([0-9]+)[/]([0-9]+)[/]([0-9]+).*?([0-9]+)[:]([0-9]+).*?')

	if reg.search(timeDate):
		k = reg.search(timeDate).group(4)
		k = int(k)
		k = k + chieu
		k = str(k)
		if len(k) == 1:
			k = '0' + k
		timeDate = k + ':' + numberFomat(reg.search(timeDate).group(5)) + ' ' + numberFomat(reg.search(timeDate).group(1)) + '-' + numberFomat(reg.search(timeDate).group(2)) + '-' + checkYear(reg.search(timeDate).group(3))
	else:
		print 'loi khong tach duoc thoi gian'
		return ''
	return timeDate
